//根据faiss生成的聚类信息,以及diskann的图结构，生成VectorSSD的图结构
//Compile: gcc gen_vectorssd_index.c vectorssd_index.c ../min_list/min_list.c -lm -o gen_vectorssd_index
/*Run(PVE): 
./gen_vectorssd_index \
/home/gary/Code/DiskANN/build/data/sift/base_500Mindex/_disk.index \
/home/gary/Code/vector-ssd/faiss_test/vector_cluster_id.bin \
vectorssd._disk.index
*/
/*Run(WFS): 
./gen_vectorssd_index \
/home/gary/Code/DiskANN/build/data/sift/base_500Mindex/_disk.index \
/home/gary/Code/vector-ssd/faiss_test/vector_cluster_id.bin \
vectorssd._disk.index
*/
//生成形成统一的VectorSSD格式。VectorSSD采用16K对齐而不是4K对齐，和闪存粒度匹配

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "vectorssd_index.h"

int main(int argc, void* argv[]){
    if(argc < 4){
        printf("Exec error, path is not given\n");
        printf("Usage: ./gen_vectorssd_index <DiskANN Index File Path> <Cluster File Path> <Output File>\n");
        return -1;
    }

    char* diskann_index_file = argv[1];
    char* cluster_file = argv[2];
    char* output_file = argv[3];

    printf("diskann index file:%s\n",diskann_index_file);
    printf("cluster file:%s\n",cluster_file);
    printf("output file:%s\n",output_file);

    //开始读取index文件。
    INDEX_HEAD head_raw; //之所以要分开是因为要用raw的头来读取原始DiskANN的索引文件，update的用来写入vectorssd的索引文件
    INDEX_HEAD head_update;
    //加载该索引的信息 
    FILE *fp = fopen(diskann_index_file, "r");
    load_index_file_SIFT1M(fp, &head_raw);
    memcpy(&head_update, &head_raw, sizeof(INDEX_HEAD));

    //修改文件头
    head_update.bytes_per_vector += head_update.neighbor_count * 2 - 4;//重新计算每个向量的空间大小。-4是因为去掉了每个向量前面标识有多少个邻居的字段
    head_update.vector_per_block = PAGE_SIZE / head_update.bytes_per_vector; //重新计算每个page可以存放多少个向量
    int page_count = (head_update.total_vectors / head_update.vector_per_block)+1;
    head_update.total_index_file_size = (1+page_count)*PAGE_SIZE;
    head_update.block_size = PAGE_SIZE;

    //每个块末尾需要补多少0
    int reserve_zero_count = PAGE_SIZE-(head_update.vector_per_block)*(head_update.bytes_per_vector);

    printf("Updated head:\n");
    printf("|-bytes_per_vector:%lu\n", head_update.bytes_per_vector);
    printf("|-vector_per_block:%lu\n", head_update.vector_per_block);
    printf("|-page_count:%d\n", page_count);
    printf("--total_index_file_size:%lu\n", head_update.total_index_file_size);

    //写入文件头
    FILE* output_fp = fopen(output_file, "w");
    fwrite(&head_update,sizeof(INDEX_HEAD),1,output_fp);
    char zeros[PAGE_SIZE] = {0};
    fwrite(zeros,PAGE_SIZE-sizeof(INDEX_HEAD),1,output_fp); //填0

    //打开聚类信息文件
    FILE* cluster_fp = fopen(cluster_file, "r");
    unsigned short* clusters = malloc(sizeof(short)*head_raw.total_vectors);
    //读取聚类.这里只能尽量一次性读到内存，因为后面是纯随机访问，按需读取开销极高。vid的聚类即为clusters[vid]
    fread(clusters, sizeof(short), head_raw.total_vectors, cluster_fp);

    //读取原始向量
    NODE_FLOAT* node_float_buf = create_node_float_buffer(&head_raw, DISKANN_FORMAT);
    unsigned short* cluster_buf = malloc(sizeof(short) * head_raw.neighbor_count); //缓存聚类的buffer，查询到的聚类先写到这个buf里面再写入降低IO次数

    FILE* diskann_index_fp = NULL;
    diskann_index_fp = fopen(diskann_index_file, "r");
    for(int i=0; i<head_raw.total_vectors; i++){
        if(i%100000==0){
            printf("Writing %dth vector\n",i);
        }

        read_node_float_fromfile(diskann_index_fp, &head_raw, i, node_float_buf);
        // print_node_float(&head_raw, node_float_buf);

        //加上聚类信息后写入
        fwrite(node_float_buf->features, head_raw.bytes_per_dim, head_raw.dim, output_fp); //写入原始向量
        fwrite(node_float_buf->neighbors, 4, head_raw.neighbor_count, output_fp); //写入邻居节点id

        //查询每个邻居的聚类
        for(int i=0; i<head_raw.neighbor_count; i++){
            cluster_buf[i] = clusters[node_float_buf->neighbors[i]];
        }
        fwrite(cluster_buf, sizeof(short), head_raw.neighbor_count, output_fp); //写入邻居节点的聚类

        if((i+1)%head_update.vector_per_block == 0){ //该补0了
            fwrite(zeros,reserve_zero_count,1,output_fp);
        }
    }

    //写完后在最后一个文件末尾补0，好16K对齐。
    int remain_vector_count = head_update.vector_per_block - (head_update.total_vectors % head_update.vector_per_block);
    if(remain_vector_count == head_update.vector_per_block){ //如果刚好写完最后一页没有剩余空间
        fwrite(zeros,reserve_zero_count,1,output_fp);
    }
    else{
        reserve_zero_count += head_update.bytes_per_vector * remain_vector_count;
        fwrite(zeros,reserve_zero_count,1,output_fp);
    }
    

    destroy_node_float(node_float_buf);

    free(clusters);
    free(cluster_buf);
    fclose(fp);
    fclose(diskann_index_fp);
    fclose(output_fp);
    return 0;
}